﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace EncodingFriendlySpider
{
    class Program
    {
        static void Main(string[] args)
        {
            var reader = new StreamReader(Settings.Default.HtmlCacheIndex);
            while (!reader.EndOfStream) 
            {
                var url = reader.ReadLine();
                var file = reader.ReadLine();
                var tofile = Settings.Default.NewHtmlDir + new FileInfo(file).Name;
                try
                {
                    var downloader = new HttpDownloader(url);
                    var html = downloader.GetPage();
                    using (var writer = new StreamWriter(tofile, false, Encoding.UTF8))
                    {
                        writer.Write(html);
                    }
                }
                catch (Exception)
                {
                    Console.WriteLine(url);
                    File.Copy(file, tofile);
                }
                
            }
        }
    }
}
